/**
 * 
 */
package net.looker.monitor.crawler;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import net.looker.config.ConfigManager;
import net.looker.work.CallBack;

/**
 * 网页抓取器
 * 用于抓取网页
 * @author solosky
 *
 */
public class WebCrawler extends Crawler 
{
	/**
	 * 页面对象
	 */
	private String html;
	/**
	 * 页面编码类型
	 */
	private String charset;
	
	/**
	 * 构造函数
	 * @param String 页面对象
	 * @throws MalformedURLException 
	 */
	public WebCrawler(String url, String charset) throws MalformedURLException 
	{
		super(url);
		this.html     = new String();
		this.charset  = charset;
		this.finished =  false;
	}
	
	/**
	 * 构造函数
	 * @param String 页面对象
	 * @throws MalformedURLException 
	 */
	public WebCrawler(String url, String charset,CallBack cb) throws MalformedURLException 
	{
		super(url, cb);
		this.html     = new String();
		this.charset  = charset;
		this.finished =  false;
	}
	
	
	/**
	 * 构造函数
	 * @throws MalformedURLException 
	 */
	public WebCrawler(String url) throws MalformedURLException
	{
		super(url);
		this.html = new String();
		this.charset = "utf-8";
	}

	/**
	 * 网页抓取类
	 * 返回是否抓取成功
	 * @throws IOException 
	 * @throws CrawlerException 
	 */
	@Override
	public void crawl() throws IOException, CrawlerException
	{
			System.out.println("Start fetching "+url.toString());
			openURL();
			//分析URL;
			html = getHtml();
			
			//设置抓取成功的标志
			finished  = true;	
	}
	
	/**
	 * 返回此URL的网页内容
	 * 去除了空白字符，便于后面的正则表达式提取信息
	 * @return
	 * @throws IOException
	 */
	public String getHtml() throws IOException
	{
		BufferedReader r = new BufferedReader(new InputStreamReader(conn.getInputStream(),charset));
		//System.out.println(getCharset());
		StringBuffer s = new StringBuffer();
		String line = null;
		while ( (line = r.readLine())!=null )
		{
			//去除空白字符 TODO 可以用正则表达式去除 。。。
			line = line.replaceAll("\t", "");
			line = line.replaceAll("\r", "");
			line = line.replaceAll("\n", "");
			s.append(line);
		}
		return s.toString();
	}


	/* (non-Javadoc)
	 * @see net.looker.monitor.crawler.Crawler#getCrawled()
	 */
	@Override
	public Object getCrawled() 
	{
		return html;
	}
}
